Mercedes-Benz EDA Notebook

Import Packages

library(ggplot2)
library(plotly)
library(data.table)

Import Data and look at first couple records

setwd("F:/kaggle/Mercedes-Benz")
sample_submission <- fread("data/sample_submission.csv")
train <- fread("data/train.csv")
test <- fread("data/test.csv")
train[1:2,]

# 1 - ID
# 2 - Response Varuable (Time in Seconds on Test Stand)
# 3-10  String Option Codes
# 11-385 0/1 values based on option codes
# X7 and X9 are missinf for some reason.

Calculate Mean and SD for Response Variable (Y). This is the time (sec) for a vehicle on the MB test station.

mean(train[,y])

[1] 100.6693

sd(train[,y])

[1] 12.67938

Create plots for the Response Variable. Sorted by ID.

plot_ly(y=train$y, type="scatter")

Create plots for the Response Variable. Sorted by the Response Variable.

plot_ly(y=train[order(y),]$y, type="scatter")

Create histogram for the Response Variable.

 plot_ly(x=train$y, type="histogram")

Create histogram for the LOG(Response Variable).

 plot_ly(x=log(train$y), type="histogram")

Frequency tables for the first 10 variables (All with String Codes)

train[,.N,by=X0][order(-N)]

train[,.N,by=X1][order(-N)]

train[,.N,by=X2][order(-N)]

train[,.N,by=X3][order(-N)]

train[,.N,by=X4][order(-N)]

train[,.N,by=X5][order(-N)]

train[,.N,by=X6][order(-N)]

train[,.N,by=X8][order(-N)] #where is 7

train[,.N,by=X10][order(-N)] #where is 9

Frequency plots for the first 10 variables (All with String Codes)

# column_names_for_option_plots_string_codes <- colnames(train)[3:5] #3:12
# for(i in column_names_for_option_plots_string_codes){
#   x_values <- as.data.frame(train[,.N,by=i][order(-N)])[,i]
#   y_values <- as.data.frame(train[,.N,by=i][order(-N)])[,'N']
#   plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar")
##Couldn't get loop plotting to work => https://github.com/ropensci/plotly/issues/273
#Plotting manually
column_names_for_option_plots_string_codes <- colnames(train)[3:10] #3:10
i<-1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

i<-i+1
current_colname <- column_names_for_option_plots_string_codes[i]
x_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,current_colname]
y_values <- as.data.frame(train[,.N,by=current_colname][order(-N)])[,'N']
plot_ly(x=as.list(x_values),y=as.list(y_values), type="bar") %>% layout(title=current_colname)

Frequency table for the remaining variables (All with 0/1 Coding)

column_names_for_summary <- colnames(train)[11:length(colnames(train))]
Variable <- c()
N_0 <- c()
N_1 <- c()
for(i in column_names_for_summary){
  j=1
  Variable <- c(Variable,i)
  N_0 <- c(N_0,train[,.N,by=i][1]$N)
  N_1 <- c(N_1,train[,.N,by=i][2]$N)
  j <- j+1
}
N_0[is.na(N_0)==TRUE] <- 0
N_1[is.na(N_1)==TRUE] <- 0
summary_results <- as.data.frame(cbind(Variable,N_0,N_1), stringsAsFactors = FALSE)
summary_results

After Looking at Kaggle, checked for Duplicate fileds - Added Data to Frequency table for the remaining variables (All with 0/1 Coding).

train_2 <- train[, !duplicated(t(train))]
duplicate_column <- c()
for(i in duplicated(t(train))){
  duplicate_column <- c(duplicate_column,i)
}
duplicate_column <- duplicate_column[11:length(duplicate_column)]
summary_results$duplicate_column <- duplicate_column
summary_results_decreasing <- summary_results[order(-N_1),]
summary_results

summary_results_decreasing

List of Duplicated Columns

summary_results_decreasing[(summary_results_decreasing$duplicate_column==TRUE),]

Correlation Matrix of Features after removing duplicates

library(ggcorrplot)

train_2 <- train[, !duplicated(t(train))] #remove duplicated fields ... from raddar@Kaggle => https://www.kaggle.com/c/mercedes-benz-greener-manufacturing/discussion/34006
ggcorrplot(cor(train_2), p.mat = cor_pmat(train_2), hc.order=TRUE, type='lower')